package com.guiji.quartz.domain;

import cn.hutool.core.util.ReUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.task.DataSaveUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.CookieManager;
import java.net.CookieStore;
import java.net.HttpCookie;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: gaoX
 * @date 2022/1/13 16:28
 */
public class Kr36 {
	private static final Logger log = LoggerFactory.getLogger(Kr36.class);

	public static void mainMethod() throws Exception {
		HttpResponse response = null;
		List<HttpCookie> cookies = null;
		try {
			response = HttpRequest.get("https://36kr.com/information/travel")
				.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
				.header("Accept-Encoding", "gzip, deflate, br")
				.header("Host", "36kr.com")
				.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36")
				.setConnectionTimeout(15000)
				.execute();
			if (response.getStatus() == 302) {
				cookies = response.getCookies();
				String new_url = response.header("Location");
				if (null != cookies && cookies.size() > 0) {
					CookieManager ccm = new CookieManager();
					CookieStore cookieStore = ccm.getCookieStore();
					for (HttpCookie cookie : cookies) {
						URL domain = new URL("https://36kr.com");
						cookieStore.add(domain.toURI(), cookie);
					}
					HttpRequest.setCookieManager(ccm);
				}

				response = HttpRequest.get(new_url)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate, br")
					.header("Host", "36kr.com")
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36")
					.setConnectionTimeout(15000)
					.execute();
			}
		} catch (Exception e) {
			log.error("代理失败1" + e);
			return;
		}
		String resp = response.body();
		Set<String> resUrl = getUrl(resp);
		getContent(resUrl);

	}

	private static void getContent(Set<String> listContent) {
		for (String url : listContent) {
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc;
			JXDocument jxDocument;
			try {
				HttpResponse response = HttpRequest.get(url).setConnectionTimeout(15000)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate, br")
					.header("Host", "36kr.com")
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36")
					.execute();
				doc = response.body();
				jxDocument = new JXDocument(doc);
			} catch (Exception e) {
				log.info("代理失败2" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;
			}

			List<Object> newImgs = new ArrayList<>();
			try {
				newImgs = jxDocument.sel("//div[@class='common-width content articleDetailContent kr-rich-text-wrapper']/p/img/@src");
			} catch (Exception e) {
				log.error("获取图片失败");
			}

			Document document = null;
			String cont = null;
			StringBuffer content = new StringBuffer();
			try {
				for (int i = 1; i <= newImgs.size(); i++) {
					doc = doc.replaceAll(newImgs.get(i-1).toString(), "<p>#" + i + "#</p>");
				}
				document = Jsoup.parse(doc);
				cont = document.getElementsByClass("common-width content articleDetailContent kr-rich-text-wrapper").get(0).toString().replaceAll("\n", "");
				ReUtil.findAll("<p>.*?</p>", cont, 0).forEach(s -> {
					content.append(s.replaceAll("<.*?>", "").replaceAll("&nbsp;", " ") + "$$$");
				});
			} catch (Exception e) {
				continue;
			}


			Date date = new Date();

			if (content.length() > 50000 || content.toString().replaceAll("本文来自.*?\\$\\$\\$", "").replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			if (newImgs.size()<1){
				continue;
			}
			try {
				DataSaveUtil.saveData(title, content.toString(), newImgs, url, date, "https://36kr.com/information/travel", "汽车新闻-36");
				log.info(Thread.currentThread().getName() + "-----------------汽车新闻-36----------------");
			} catch (Exception e) {
				continue;
			}

		}
	}

	private static Set<String> getUrl(String content) {

		Set<String> res = new HashSet<>();
		String pattern = "/p/[0-9]{16}";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content.replaceAll("\n", "").replaceAll("<script>.*?</script>", ""));
		while (m.find()) {
			res.add("https://36kr.com" + m.group());
		}
		return res;
	}

	public static void main(String[] args) throws Exception {
//		Set<String> res = new HashSet<>();
//		res.add("https://36kr.com/p/1577598441310978");
//		getContent(res);
		mainMethod();
	}
}
